package cn.cas.mango.service;

import cn.cas.mango.config.Mapping;
import cn.cas.mango.dbscan.Data;
import cn.cas.mango.dto.param.GlobalParam;
import cn.cas.mango.dto.param.HitNews;
import cn.cas.mango.dto.param.HotKeywords;
import cn.cas.mango.dto.param.News;
import cn.cas.mango.dto.param.NewsCluster;
import cn.cas.mango.dto.param.neo4j.NodeEntity;
import cn.cas.mango.singlePass.SinglePass;
import cn.cas.mango.singlePass.SinglePassData;
import cn.cas.mango.util.CheckParam;
import cn.cas.mango.util.GetArea;
import cn.cas.mango.util.Pair;
import cn.cas.mango.util.TimeUtil;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest;
import co.elastic.clients.json.JsonData;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;


import java.io.InputStreamReader;
import java.io.StringReader;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Service
@Slf4j
public class PrintJson {
  private final ObjectMapper objectMapper = new ObjectMapper();
  @Value("${elasticsearch.minNeighbors}")
  private Integer minNeighbors;
  @Autowired
  private RequestService requestService;

  @Autowired
  private GetJson getJson;
  // 最小的keywords大小
  @Autowired
  private Neo4jService neo4jService;
  public String[] blackListPath = {"normal.txt", "media.txt"};

  @Autowired
  private ElasticsearchClient client;

  @Value("${elasticsearch.newIndex}")
  private String INDEX;

  @Value("${elasticsearch.clusterIndex}")
  private String CLUSTERINDEX;

  @Value("${elasticsearch.single.threshold}")
  private double singlePointsThreshold;

  private boolean createIndex(String index, String type) {
    boolean exists;
    try {
      exists = client.indices().exists(x -> x.index(index)).value();
      if (exists) {
        client.indices().delete(new DeleteIndexRequest.Builder().index(index).build());
      }
      CreateIndexRequest createIndexRequest = CreateIndexRequest.of(x -> x.index(index).mappings(m -> m.withJson(new StringReader(Mapping.getMapping(type))))
          .settings(s -> s.withJson(new StringReader(Mapping.setting()))));
      client.indices().create(createIndexRequest
//            .settings(s->s.withJson(new StringReader(Mapping.setting())))
      );
      return exists;
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }


  /**
   * 将不属于同一簇的邻居剔除
   *
   * @param map2Score,             邻居相似度
   * @param news2Cast，簇id关于news的映射
   * @return 剔除后的邻居相似度map
   */
  private Map<String, Map<String, Double>> filterNeighbors(Map<String, Map<String, Double>> map2Score,
                                                           Map<String, News> news2Cast, int type) {
    Map<String, String> member2Cluster = new HashMap<>();
    for (String key : news2Cast.keySet()) {
      News news = news2Cast.get(key);
      String clusterId;
      if (type == 1) {
        clusterId = news.getCluster_id1();
      } else if (type == 3) {
        clusterId = news.getCluster_id3();
      } else {
        clusterId = news.getCluster_id5();
      }
      member2Cluster.put(key, clusterId);
    }
    HashMap<String, Map<String, Double>> ret = new HashMap<>();
    for (String key : map2Score.keySet()) {
      ret.put(key, new HashMap<>());
      Map<String, Double> stringDoubleMap = map2Score.get(key);
      for (String neighbor : stringDoubleMap.keySet()) {
        if (member2Cluster.get(key).equals(member2Cluster.get(neighbor))) {
          ret.get(key).put(neighbor, stringDoubleMap.get(neighbor));
        }
      }
    }
    return ret;
  }

  public Set<String> getBlackListSet() {
    Set<String> blackListSet = new HashSet<>();
    Arrays.stream(blackListPath).forEach(x -> {
      try {
        blackListSet.addAll(getBlackList(x));
      } catch (IOException e) {
        log.info("blacklist file is null or in wrong format!");
      }
    });
    return blackListSet;
  }

  private File[] getSortedFile(File[] files) {
    List<Pair<String, File>> list = new ArrayList<>();
    for (File file : files) {
      list.add(new Pair<>(file.getName(), file));
    }
    list.sort(Comparator.comparing(Pair::getKey));
    return list.stream().map(Pair::getValue).toArray(File[]::new);
  }

  private List<SinglePassData> wrapSinglePassData(List<HitNews> list) {
    return list.stream().map(x -> {
      List<JSONObject> keywords = (List<JSONObject>) x.getKeywords();

      String id = x.getNm();
      return new SinglePassData(id,
          new LinkedHashSet<>(keywords.stream().map(y -> (String) y.get("name")).collect(Collectors.toList())));
    }).collect(Collectors.toList());
  }

  private List<HitNews> removeSingle(List<HitNews> hitNewsList) {
    if (singlePointsThreshold == -1) return hitNewsList;
    List<SinglePassData> singlePassDataList = wrapSinglePassData(hitNewsList);
    Set<String> run = new LinkedHashSet<>(new SinglePass(singlePassDataList).run(singlePointsThreshold));
    return hitNewsList.stream().filter(x -> run.contains(x.getNm())).collect(Collectors.toList());

  }

  public List<String> getBlackList(String filePath) throws IOException {
    FileInputStream fis = new FileInputStream(filePath);
    List<String> blackLists = new ArrayList<>();
    //Construct BufferedReader from InputStreamReader
    BufferedReader br = new BufferedReader(new InputStreamReader(fis));

    String line = null;
    while ((line = br.readLine()) != null) {
      blackLists.add(line.trim());
    }

    br.close();
    fis.close();
    return blackLists;
  }

  public void main(GlobalParam globalParam) throws IOException {
    CheckParam.checkGlobal(globalParam);
    if (createIndex(INDEX, "news")) {
      log.info("索引news存在，已删除后重新建立");
    } else {
      log.info("索引news已建立");
    }
    if (createIndex(CLUSTERINDEX, "cluster")) {
      log.info("索引cluster存在，已删除后重新建立");
    } else {
      log.info("索引cluster已建立");
    }

    List<List<News>> lists2 = getJson.main(globalParam);

    Set<String> blackListSet = getBlackListSet();

    log.info("query param: {}", globalParam);
    List<News> temp = new ArrayList<>();
    lists2.forEach(l -> temp.addAll(l));
    List<List<News>> lists = new ArrayList<List<News>>() {{
      add(temp);
    }};

    int i = 0;
    TimeUtil timeUtil = new TimeUtil();
    timeUtil.setStart();
    for (List<News> list : lists) {

      Map<String, News> newsMap = new HashMap<>();
      List<HitNews> hitNewsListBefore = new ArrayList<HitNews>() {{
        for (Object o : list) {
          News news = JSON.parseObject(JSON.toJSONString(o), News.class);
          news = news.filterKeywords(blackListSet);
          newsMap.put(news.getNm(), news);
          HitNews hitNews = new HitNews(0.0);
          BeanUtils.copyProperties(news, hitNews);
          add(hitNews);
        }
      }};
      // 局部去重

      List<HitNews> hitNewsList = removeSingle(removeDuplicates(hitNewsListBefore));
      Map<String, HitNews> map = new HashMap<>();
      for (HitNews hitNews : hitNewsList) {
        map.put(hitNews.getNm(), hitNews);
      }
//          SameNewsService sameNewsService1 = new SameNewsService(hitNewsList, SortOrder.Desc, "time",
//              globalParam.getPageSize(), globalParam.getAddScore());
//          Map<String, Map<String, Object>> clusters1 = sameNewsService1.getSameNewsList(globalParam.getSimParam(), globalParam.getV(), globalParam.getMaxClusterSize(), 2);
//          Map<String, Map<String, Double>> map2Score1 = sameNewsService1.getMap2Score();
      // 去除绝对孤立点

      SameNewsService sameNewsService3 = new SameNewsService(hitNewsList, SortOrder.Desc, "time",
          globalParam.getPageSize(), globalParam.getAddScore(), 0, new LinkedHashMap<>());


      Map<String, Map<String, Object>> clusters3 = sameNewsService3.getSameNewsList(globalParam.getSimParam(),
          globalParam.getV(), globalParam.getMaxClusterSize(), minNeighbors);
      Map<String, Map<String, Double>> map2Score3 = getScore(clusters3, globalParam.getAddScore());

//          Map<String, Map<String, Double>> map2Score3 = sameNewsService3.getMap2ExpandedScore();

//          SameNewsService sameNewsService5 = new SameNewsService(hitNewsList, SortOrder.Desc, "time",
//              globalParam.getPageSize(), globalParam.getAddScore());
//          Map<String, Map<String, Object>> clusters5 = sameNewsService5.getSameNewsList(globalParam.getSimParam(), globalParam.getV(), globalParam.getMaxClusterSize(), 4);
//          Map<String, Map<String, Double>> map2Score5 = sameNewsService5.getMap2ExpandedScore();

//          setClusterId(clusters1, newsMap, 1);
      setClusterId(clusters3, newsMap, 3);
//          setClusterKeywords(newsMap, clusterKeywords);
//          setClusterId(clusters5, newsMap, 5);

//          map2Score1 = filterNeighbors(map2Score1, newsMap, 1);
      map2Score3 = filterNeighbors(map2Score3, newsMap, 3);
//          map2Score5 = filterNeighbors(map2Score5, newsMap, 5);
      // 写入到es

      List<BulkOperation> operations = newsMap.values().stream().map(news ->
          BulkOperation.of(b -> b
              .index(y -> {
                    try {
                      return y
                          .index(INDEX)
                          .id(news.getNm())
                          .document(JsonData.fromJson(objectMapper.writeValueAsString(news)));
                    } catch (JsonProcessingException e) {
                      throw new RuntimeException(e);
                    }
                  }
              )
          )).collect(Collectors.toList());
      requestService.bulkWrite(operations);

      addSelf(clusters3);
      List<BulkOperation> clusterOperations = wrapCluster(clusters3).stream()
          .map(news ->
              BulkOperation.of(b -> b
                      .index(y -> {
                            try {
                              return y
                                  .index(CLUSTERINDEX)
//                                  .id(news.getId())
                                  .document(JsonData.fromJson(objectMapper.writeValueAsString(news)));
                            } catch (JsonProcessingException e) {
                              throw new RuntimeException(e);
                            }
                          }
                      )
              )).collect(Collectors.toList());
      requestService.bulkWrite(clusterOperations);

      log.info("part {} finished", ++i);
      if (globalParam.getToGraph()) {
        log.info("map in start");
        TimeUtil timeUtil1 = new TimeUtil();
        timeUtil1.setStart();
        write2Neo4j(map2Score3, map);
        log.info("map end, cost {}s", timeUtil1.getCost());
      }
    }

    log.info("total cost " + timeUtil.getCost() + "s");
  }


  private void addSelf(Map<String, Map<String, Object>> clusters) {
    for (String key : clusters.keySet()) {
      Map<String, Object> stringObjectMap = clusters.get(key);
      List<HitNews> like = (List<HitNews>) stringObjectMap.get("like");
      HitNews hitNews = (HitNews) stringObjectMap.get("self");
      like.add(hitNews);
    }
  }

  private Map<String, Map<String, Double>> getScore(Map<String, Map<String, Object>> clusters,
                                                    boolean addScore) {
    Map<String, Map<String, Double>> ret = new HashMap<>();
    for (String clusterId : clusters.keySet()) {
      Map<String, Object> stringObjectMap = clusters.get(clusterId);
      List<HitNews> total = new ArrayList<>();
      total.add((HitNews) stringObjectMap.get("self"));
      total.addAll((List<HitNews>) stringObjectMap.get("like"));
      if (total.size() != 1)
        getScoreMap(ret, total, addScore);
    }
    return ret;
  }

  private Map<String, Map<String, Double>> getClusterKeywords(Map<String, Map<String, Object>> clusters) {
    Map<String, Map<String, Double>> ret = new HashMap<>();
    for (String clusterId : clusters.keySet()) {
      Map<String, Object> stringObjectMap = clusters.get(clusterId);
      Map<String, Integer> clusterKeywords = (Map<String, Integer>) stringObjectMap.get("cluster_keywords");
      Map<String, Double> newClusterKeywords = new LinkedHashMap<>();
      if (!clusterKeywords.isEmpty()) {
        double total = clusterKeywords.values().stream().reduce((a, b) -> a + b).get();
        for (String key : clusterKeywords.keySet()) {
          newClusterKeywords.put(key, clusterKeywords.get(key) / total);
        }
      }
      ret.put(clusterId, newClusterKeywords);
    }
    return ret;
  }

  private void getScoreMap(Map<String, Map<String, Double>> ret,
                           List<HitNews> total, boolean addScore) {

    for (int i = 0; i < total.size(); i++) {
      HitNews hitNewsI = total.get(i);
      Data dataI = new Data(hitNewsI.getNm(), new HotKeywords(hitNewsI.getKeywords()), addScore);
      for (int j = 0; j < total.size(); j++) {
        HitNews hitNewsJ = total.get(j);
        Data dataJ = new Data(hitNewsJ.getNm(), new HotKeywords(hitNewsJ.getKeywords()), addScore);
        if (i != j) {
          if (!ret.containsKey(hitNewsI.getNm())) {
            ret.put(hitNewsI.getNm(), new HashMap<>());
          }
          ret.get(hitNewsI.getNm()).put(hitNewsJ.getNm(),
              dataI.calSim(dataJ));
        }
      }
    }
  }

//    private void setClusterKeywords(Map<String, News> newsMap,
//                                    Map<String, Map<String, Double>> clusterKeywords) {
//      for (String key: newsMap.keySet()) {
//        News news = newsMap.get(key);
//        String clusterId3 = news.getCluster_id3();
//        if (clusterKeywords.containsKey(clusterId3)) {
//          news.setClusterKeywords(wrapMap(clusterKeywords.get(clusterId3)));
//        }
//      }
//    }

  private Map<String, Set<String>> getClusterEnt(Map<String, Map<String, Object>> clusters, String type) {
    Map<String, Set<String>> ret = new HashMap<>();
    for (String clusterId : clusters.keySet()) {
      Map<String, Object> stringObjectMap = clusters.get(clusterId);
      Set<String> clusterEnt = (Set<String>) stringObjectMap.get(type);
      ret.put(clusterId, clusterEnt);
    }
    return ret;
  }

  private List<NewsCluster> wrapCluster(Map<String, Map<String, Object>> clusters3) {
    Map<String, Map<String, Double>> clusterKeywords = getClusterKeywords(clusters3);
    Map<String, Set<String>> clusterEntitiesPerson = getClusterEnt(clusters3, "cluster_entities_person");
    Map<String, Set<String>> clusterEntitiesPlace = getClusterEnt(clusters3, "cluster_entities_place");
    Map<String, Set<String>> clusterEntitiesOrganization = getClusterEnt(clusters3, "cluster_entities_organization");
    Map<String, Set<String>> clusterEntities = getClusterEnt(clusters3, "cluster_entities");
    List<NewsCluster> ret = new ArrayList<>();
    for (String key : clusterKeywords.keySet()) {
      Map<String, Double> keywords = clusterKeywords.get(key);
      List<HitNews> like = (List<HitNews>) clusters3.get(key).get("like");
      Set<String> entitiesPerson = clusterEntitiesPerson.get(key);
      Set<String> entitiesPlace = clusterEntitiesPlace.get(key);
      Set<String> entitiesOrganization = clusterEntitiesOrganization.get(key);
      Set<String> entities = clusterEntities.get(key);
      ret.add(new NewsCluster(1.0, key, "0", wrapMap(keywords),
          entities, entitiesPlace,
          entitiesOrganization, entitiesPerson, "2020-01-01",
          like.stream().map(News::getNm).collect(Collectors.toList()), ((HitNews) clusters3.get(key).get("self")).getSjmc()));
    }
    return ret;
  }

  private List<Map<String, Object>> wrapMap(Map<String, Double> clustersKeywords) {
    List<Map<String, Object>> ret = new ArrayList<>();
    for (String key : clustersKeywords.keySet()) {
      Double score = clustersKeywords.get(key);
      ret.add(new HashMap<String, Object>() {{
        put("name", key);
        put("score", score);
      }});
    }
    return ret;
  }

  private void setClusterId(Map<String, Map<String, Object>> clusters,
                            Map<String, News> newsMap,
                            int type) {
    Map<String, String> preparedUpdatedIds = new HashMap<>();
    for (String key : clusters.keySet()) {
      List<String> id2Object = new ArrayList<>();
      id2Object.add(((HitNews) clusters.get(key).get("self")).getNm());
      Stream like = ((List) clusters.get(key).get("like")).stream().map(x -> ((HitNews) x).getNm());
      id2Object.addAll((List<String>) like.collect(Collectors.toList()));
      for (String id : id2Object) {
        preparedUpdatedIds.put(id, key);
      }
    }
//      try {
//        objectMapper.writeValue(new File("temp.json"),clusters);
//      } catch (IOException e) {
//        throw new RuntimeException(e);
//      }
    for (String newsId : preparedUpdatedIds.keySet()) {
      String clusterId = preparedUpdatedIds.get(newsId);
      // 放置国别
      News news = newsMap.get(newsId);
      if (news.getSjms() == null) {
        log.info("空内容{}", news);
      }
      news.setCountryOrArea(GetArea.getOneArea(news.getSjms()));
      // 处理特殊领域
      news.processLy();
      if (type == 1) {
        newsMap.get(newsId).setCluster_id1(clusterId);
      } else if (type == 3) {
        newsMap.get(newsId).setCluster_id3(clusterId);
      } else {
        newsMap.get(newsId).setCluster_id5(clusterId);
      }
    }
  }

  private void write2Neo4j(Map<String, Map<String, Double>> map2Score,
                           Map<String, HitNews> map) {
    for (String key : map2Score.keySet()) {
      Map<String, Double> stringDoubleMap = map2Score.get(key);
      for (String neigh : stringDoubleMap.keySet()) {
        map2Score.get(neigh).remove(key);
      }
    }

    neo4jService.clearAll();
    Map<String, NodeEntity> id2Node = new HashMap<>();
    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
    for (String id : map2Score.keySet()) {
      HitNews hitNews = map.get(id);
      LocalDate date;
      try {
        date = LocalDate.parse(hitNews.getFssj(), formatter);
      } catch (Exception e) {
        date = LocalDate.parse("2010-01-01", formatter);
      }
      id2Node.put(id,
          new NodeEntity(id, hitNews.getSjmc(), hitNews.getSjms(), date, hitNews.getSsly(), hitNews.getLyss(),
              GetArea.getArea(hitNews.getSjms()), new ArrayList<>(),
              hitNews.getCluster_id3()));

    }

    for (String id : id2Node.keySet()) {
      NodeEntity node = id2Node.get(id);
      Map<String, Double> relations = map2Score.get(id);

      node.addRelationShip(relations.keySet().stream().map(id2Node::get).collect(Collectors.toList()),
          new ArrayList<>(relations.values()));
    }
    for (String id : map.keySet()) {
      if (!id2Node.containsKey(id)) {
        HitNews hitNews = map.get(id);
        LocalDate date;
        try {
          date = LocalDate.parse(hitNews.getFssj(), formatter);
        } catch (Exception e) {
          date = LocalDate.parse("2010-01-01", formatter);
        }
        id2Node.put(id,
            new NodeEntity(id, hitNews.getSjmc(), hitNews.getSjms(), date, hitNews.getSsly(), hitNews.getLyss(),
                GetArea.getArea(hitNews.getSjms()), new ArrayList<>(), hitNews.getCluster_id3()));
      }
    }
    neo4jService.createOrUpdateNode(id2Node.values());

  }

  private List<HitNews> removeDuplicates(List<HitNews> hitNewsList) {
    Map<String, HitNews> hitNewsMap = new HashMap<String, HitNews>() {{
      for (HitNews hitNews : hitNewsList) {
        put(hitNews.getNm(), hitNews);
      }
    }};
    return new ArrayList<>(hitNewsMap.values());
  }
}
